package spider.linkedin;

import java.io.File;
import java.io.FileOutputStream;
import java.io.IOException;
import java.io.InputStream;
import java.nio.file.Files;
import java.nio.file.Path;
import java.nio.file.Paths;
import java.time.LocalDate;
import java.time.format.DateTimeFormatter;
import java.util.ArrayList;
import java.util.Collections;
import java.util.List;

import org.apache.log4j.Logger;

import com.alibaba.fastjson.JSON;
import com.alibaba.fastjson.JSONArray;
import com.alibaba.fastjson.JSONObject;
import com.xiaoleilu.hutool.http.HttpException;
import com.xiaoleilu.hutool.http.HttpRequest;
import com.xiaoleilu.hutool.http.HttpResponse;
import com.xiaoleilu.hutool.io.FileUtil;
import com.xiaoleilu.hutool.io.IoUtil;
import com.xiaoleilu.hutool.log.LogFactory;
import com.xiaoleilu.hutool.util.RandomUtil;
import com.xiaoleilu.hutool.util.ThreadUtil;

public class App {
	private Logger logger = Logger.getLogger(App.class);
	private static Path workPath = null;
	private static String resumePath = "";
	private static String cookiefile = "";
	private static String taskfile = "";
	private static String idxfile = "";
	String cookie = "";

	static {
		String path = App.class.getProtectionDomain().getCodeSource().getLocation().getFile();
		if (path.startsWith("/")) {
			path = path.substring(1, path.length() - 1);
		}
		workPath = Paths.get(path).getParent();
		resumePath = workPath + File.separator + "resume";
		cookiefile = workPath + File.separator + "cookie.txt";
		taskfile = resumePath + File.separator + "task.txt";
		idxfile = resumePath + File.separator + "idx.txt";
	}

	public static void main(String[] args) throws IOException {
		App downfile = new App();
		try {
			downfile.downFiles();
		} catch (Exception e) {
			LogFactory.get().info(e);
			if (e.getMessage().contains("Server redirected too many  times")) {
				LogFactory.get().info("Cookie 失效，请重新登录账号并复制Cookie数据");
			}

		}
	}

	public List<Project> getProject() {
		String[] schools="武汉大学,中国科学技术大学,中国人民大学,湖南大学,同济大学,国防科学技术大学,吉林大学,山东大学,大连理工大学,东北大学".split(",");
		
		String url = "https://www.linkedin.com/recruiter/api/projects/searchWithStatuses?q=&view=sharedWithMe&start=0&state=A&count=1000";
		String result = HttpRequest.get(url).cookie(cookie).execute().body();
		JSONArray ja = JSON.parseObject(result).getJSONObject("result").getJSONObject("projects").getJSONArray("data");
		List<Project> projects = new ArrayList<Project>();
		for (int i = 0; i < ja.size(); i++) {
			JSONObject project = ja.getJSONObject(i);
			String id = project.getString("projectId");
			String name = project.getString("name");
			for(int k=0;k<schools.length;k++){
				if(name.contains(schools[k])){
					projects.add(new Project(id, i + name));
				}
			}
			
		}
		return projects;
	}

	public void downFiles() {
		logger.info("当前工作路径:" + App.workPath);
		List<Task> tasks = getTasks();
		LocalDate ld = LocalDate.now();
		String datestr = ld.format(DateTimeFormatter.ofPattern("yyyyMMdd"));
		int trytimes = 3;
		String sum=tasks.size()==0?"":tasks.get(tasks.size()-1).getIdx();
		for (int i = 0; i < tasks.size(); i++) {
			Task task = tasks.get(i);
			String prospectid = task.getProspectid();
			String id="";
			try {
				logger.info(String.format("开始抓取第%s/%s简历", task.getIdx(),sum));
				String url2 = "https://www.linkedin.com/cap/people/profileExportPdf/_LinkedIn_" + datestr
						+ ".pdf?memberIds=" + prospectid + ",PTS,PTS&origin=profile";
				HttpResponse resopon=HttpRequest.get(url2).cookie(cookie).timeout(10000).execute();
				if(resopon.getStatus()!=200&&resopon.getStatus()!=404){
					logger.info(resopon.getStatus()+"抓取数据失败，可能被禁用，请关闭程序并重新登录账号并复制Cookie数据,如不关闭程序，将再15分钟后重试..");
					ThreadUtil.sleep(15*60*1000);
					continue;
				}else if(resopon.getStatus()==404){
					logger.info("抓取数据失败,跳过");
					continue;
				}
				String result = resopon.body();
				logger.info(result);
				id = JSON.parseObject(result).getString("id");
				String url3 = "https://www.linkedin.com/cap/people/printStateAjax?id=" + id;
				String isready = HttpRequest.get(url3).cookie(cookie).timeout(10000).execute().body();
				// logger.info(isready);
				downFile(task, id);
				FileUtil.writeString(task.getIdx(), idxfile, "utf-8");
			} catch (Exception ex) {
				String fileUrl = "https://www.linkedin.com/cap/people/streamPdf/" + id;
				ex.printStackTrace();
				if(ex.getMessage()!=null&&ex.getMessage().toLowerCase().contains("gzip")){
					logger.info("抓取失败...请手动抓取:" + prospectid + ":" + fileUrl);
					logfail(task, fileUrl);
				}else if(ex.getMessage()!=null&&ex.getMessage().toLowerCase().contains("server redirected too many")){
					logger.info("Cookie 失效，请重新登录账号并复制Cookie数据");
					System.exit(-1);
				}else{
					if (trytimes > 0) {
						i--;
						trytimes--;
						logger.info("抓取失败重试...");
						ThreadUtil.sleep(4000);
					} else {
						trytimes = 3;
						logger.info("抓取失败,不再重试...连接写入文件");
						logfail(task, fileUrl);
					}
				}

			}
			int sleeptime = 1000 + RandomUtil.randomInt(10, 1000);
			logger.info("休息:" + sleeptime + "ms");
			ThreadUtil.sleep(sleeptime);
		}

	}

	public List<Task> getTasks() {
		List<Task> tasks = new ArrayList<Task>();
		String cookieStr = FileUtil.readString(cookiefile, "utf-8");

		if (cookieStr.trim().isEmpty()) {
			logger.info("请先按照使用说明填写cookie");
			System.exit(-1);
		} else {
			cookie = cookieStr;
		}
		File file=new File(taskfile);
		if(!file.exists()){
			try {
				file.createNewFile();
			} catch (IOException e) {
				e.printStackTrace();
				logger.info("创建task文件失败，请检查是否有相应权限");
			}
		}
		List<String> filetasks = FileUtil.readLines(taskfile, "utf-8");

		if (filetasks.size() == 0) {
			logger.info("等待抓取任务列表..");
			List<Project> projects = getProject();
			int idx = 0;
			for (int i = 0; i < projects.size(); i++) {
				Project project = projects.get(i);
				JSONArray ja = getProspectIds(project);
				logger.info("抓取:" + project.getName() + ":简历列表");
				for (int j = 0; j < ja.size(); j++) {
					String prospectid = ja.getString(j);
					Task task = new Task(String.valueOf(idx++), project.getName(), prospectid);
					tasks.add(task);
				}
				ThreadUtil.sleep(2000);
			}
			FileUtil.writeLines(tasks, taskfile, "utf-8");
			logger.info("任务列表抓取完成，抓取简历...");
			System.exit(-1);
		} else {
			logger.info("已有任务列表，继续抓取简历...");
			String idx = FileUtil.readString(idxfile, "utf-8");
			int i = idx.isEmpty() ? 0 : Integer.valueOf(idx) + 1;
			for (; i < filetasks.size(); i++) {
				tasks.add(new Task(filetasks.get(i)));
			}
		}

		return tasks;
	}

	public JSONArray getProspectIds(Project project) {
		String url = "https://www.linkedin.com/recruiter/api/projects/" + project.getId()
				+ "/profiles?count=25&start=0&fetchAllIds=true";
		int trytimes=3;
		String result="";
		while(result.equals("")&&trytimes>0){
			try{
				result = HttpRequest.get(url).cookie(cookie).timeout(20000).execute().body();
			}catch(Exception e){
				e.printStackTrace();
				trytimes--;
				logger.info("抓取失败，重试");
			}
		}
		if(result.equals("")){
			return new JSONArray();
		}
		// logger.info(result);
		JSONArray ja = JSON.parseObject(result).getJSONObject("result").getJSONArray("prospectIds");
		// logger.info(ja);
		return ja;
	}

	public void logfail(Task task, String url) {
		String log = task.getProjectName() + "," + task.getProspectid() + "," + url;
		FileUtil.appendLines(Collections.singleton(log), resumePath + File.separator + "fail.log", "utf-8");
	}

	private void downFile(Task task, String id) {
		String fileUrl = "https://www.linkedin.com/cap/people/streamPdf/" + id;
		HttpResponse response = HttpRequest.get(fileUrl).cookie(cookie).timeout(20000).execute();
		String filename = response.header("Content-Disposition").split("=")[1].replaceAll("\"", "");
		filename = task.getProspectid() + "." + filename.split("\\.")[0] + "." + RandomUtil.randomInt(2000) + ".pdf";
		logger.info(filename);
		InputStream in = HttpRequest.get(fileUrl).cookie(cookie).execute().bodyStream();
		try {
			String path = resumePath + File.separator + task.getProjectName();
			logger.info("Download size: " + in.available());
			Files.createDirectories(Paths.get(path));
			FileOutputStream file = new FileOutputStream(path + File.separator + filename);
			IoUtil.copyByNIO(in, file, IoUtil.DEFAULT_BUFFER_SIZE, null);
			file.close();
		} catch (Exception e) {
			logger.info(e.getMessage());
		}

	}
}
